Import and Filter TOC LEEF-1

Some preliminary setup

Show the code
library(LEEF.analysis)
library(LEEF.measurement.toc)
library(dplyr)

Attaching package: 'dplyr'
The following objects are masked from 'package:stats':

    filter, lag
The following objects are masked from 'package:base':

    intersect, setdiff, setequal, union
Show the code
library(tidyr)
library(ggplot2)
library(ggExtra)
options(dplyr.summarise.inform = FALSE)
Show the code
root_folder <- params$root_folder

db <- params$db
options(RRDdb = db)

unlink(file.path(root_folder, "1.pre-processed.data", "toc"))
pre_processor_toc(file.path(root_folder, "0.raw.data"), file.path(root_folder, "1.pre-processed.data"))
[1] "Log file set to ~/Desktop//1.pre-processed.data/toc/toc.log"

########################################################

Processing toc
done

########################################################
Show the code
unlink(file.path(root_folder, "2.extracted.data", "toc"))
extractor_toc(file.path(root_folder, "1.pre-processed.data"), file.path(root_folder, "2.extracted.data"))
Extracting toc
[1] "Log file set to ~/Desktop//2.extracted.data/toc/toc.log"

########################################################
Extracting toc
Processing LEEF_ 22_06_17A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_ 22_06_20A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_ 22_06_20B.txt ...
Warning in readLines(fn): incomplete final line found on
'/Users/rainerkrug/Desktop//1.pre-processed.data/toc/LEEF_ 22_06_20B.txt'
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_09_20 und 21_09_22  C.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_09_24 1-8.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_09_24 und 21_09_27 B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_09_27 restB.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_09_29t.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_01.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_04 S22 - S30.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_04 und 21_10_06A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_06.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_08.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_11.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_13.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_15 s8 - 26.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_15.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_18.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_20.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_22.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_25.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_27.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_10_29 und 21_11_01.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_11_03 und 21_11_05.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_11_08 und 21_11_10.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_11_12.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_11_17 und 21_11_19.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_11_22 und 21_11_24.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_11_26 und 21_11_29.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_12_01 und 21_12_03A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_12_03B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_12_06 und 21_12_08 1-8A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_12_08B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_12_10 und 21_12_13.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_12_15 und 21_12_17.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_12_20 und 21_12_22.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_12_24.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_12_27 und 21_12_29.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_12_31 und 22_01_03.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_21_15_10 rest 2.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_01_05 und 22_01_07A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_01_10 und 22_01_12.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_01_14 und 22_01_17.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_01_19 und 22_01_21A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_01_19 und 22_01_21B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_01_24 und 22_01_26.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_01_28 und 22_01_31A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_02 und 22_02_04.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_07B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_07D.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_09 und 22_02_11A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_14 und 22_02_16A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_16B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_18A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_18B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_21A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_23A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_25.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_02_28.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_02.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_04.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_07.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_09.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_11.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_14.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_16.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_18.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_21.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_23.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_25.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_28.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_30A.txt ...
Warning in readLines(fn): incomplete final line found on
'/Users/rainerkrug/Desktop//1.pre-processed.data/toc/LEEF_22_03_30A.txt'
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_03_30B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_01.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_04A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_04B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_06.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_08.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_11.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_13.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_15A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_15B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_18.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_20.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_22.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_25.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_27.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_04_29.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_02.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_04.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_06.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_09.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_11.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_13.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_16.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_18.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_20.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_23A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_25A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_27A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_05_30A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_01A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_03A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_06A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_08A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_10B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_13A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_15A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_20F.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_22A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_27A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_06_29B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing LEEF_22_07_04A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
done

########################################################

Read parameter and data files and determine CV and number extra measurements

Show the code
fns <- sapply(
  list.files(file.path(root_folder, "2.extracted.data", "toc"), "\\.data\\.", full.names = FALSE),
  function(fn){
    x <- strsplit(fn, "\\.")[[1]][[1]]
  }
)

data <- lapply(
    fns,
    function(fn){
      p <- read.csv(file.path(root_folder, "2.extracted.data", "toc", paste(fn, "parameter", "csv", sep = ".")))
      d <- read.csv(file.path(root_folder, "2.extracted.data", "toc", paste(fn, "data", "csv", sep = ".")))
      return(list(
        data = d, 
        parameter = p, 
        samples = max(p$samples), 
        extra_samples = max(p$extra_samples), 
        max_cv = max(p$max_cv)
      )
      )
    }
)

sapply(data, "[[", "samples") |> plot(ylab = "samples") |> print()

NULL
Show the code
sapply(data, "[[", "extra_samples") |> plot(ylab = "extra_samples") |> print()

NULL
Show the code
sapply(data, "[[", "max_cv") |> plot(ylab = "max_cv") |> print()

NULL
Show the code
samples <- sapply(data, "[[", "samples") |> min()
extra_samples <- sapply(data, "[[", "extra_samples") |> min()
max_cv <- sapply(data, "[[", "max_cv") |> max()

no_samples <- samples + extra_samples
max_col <- (2 + 3 + no_samples * 2) + 1

As the number of additional samples as well as the cv were different for some samples, we have to re-calculate conc and cv. The values which will be used are

  • samples: 2
  • extra_samples: 1
  • max_cv: 3

In case there are more extra samples, the last ones were be discarded.

Read data and discard more than 1 extra samples

Show the code
toc <- lapply(
  data,
  function(x){
    result <- x$data[,1:max_col]
    return(result)
  }
) |> do.call(what = rbind)
toc$id <- 1:nrow(toc)

Remove missing inj_type

We have the following measurements files with measurements without inj_type or NULL. These can nod be analysed and need to be excluded.

Show the code
toc %>%
  filter(inj_type == "") %>%
  group_by(filename) %>%
  summarise(n = n()) %>% 
  collect() %>%
  knitr::kable()
filename n
LEEF_ 22_06_20A 78
LEEF_ 22_06_20B 2
LEEF_21_09_24 1-8 208
LEEF_22_01_19 und 22_01_21A 196
LEEF_22_02_09 und 22_02_11A 3
LEEF_22_02_14 und 22_02_16A 3
LEEF_22_02_16B 3
LEEF_22_04_04A 54
LEEF_22_04_15A 58
LEEF_22_06_03A 6
LEEF_22_06_06A 6
LEEF_22_06_08A 6
LEEF_22_06_10B 6
LEEF_22_06_13A 6
LEEF_22_06_15A 6

These can be filtered out

Show the code
before <- nrow(toc)
toc <- toc %>%
  filter(inj_type != "")
after <- nrow(toc)
cat("Before : ", before, "\n")
Before :  17424 
Show the code
cat("Removed: ", before - after, "\n")
Removed:  645 
Show the code
cat("After  : ", after, "\n")
After  :  16779 

Remove rows from validation / null samples

In addition to the analysed samples, two more validation and null samples (H2O) were taken. Upon further consideration, these were considered as not useful and excluded from the analysis. They are saved as toc_val_samples.csv.

Show the code
toc %>%
  filter(identification == "H2O" | identification == "2.5mg/lIC") %>%
  group_by(filename) %>%
  summarise(n = n()) %>% 
  collect() %>%
  knitr::kable()
filename n
LEEF_ 22_06_17A 8
LEEF_ 22_06_20A 4
LEEF_ 22_06_20B 4
LEEF_21_09_20 und 21_09_22 C 16
LEEF_21_09_24 1-8 8
LEEF_21_09_24 und 21_09_27 B 16
LEEF_21_09_27 restB 16
LEEF_21_09_29t 16
LEEF_21_10_01 16
LEEF_21_10_04 S22 - S30 12
LEEF_21_10_04 und 21_10_06A 16
LEEF_21_10_06 16
LEEF_21_10_08 16
LEEF_21_10_11 16
LEEF_21_10_13 8
LEEF_21_10_15 8
LEEF_21_10_15 s8 - 26 4
LEEF_21_10_18 8
LEEF_21_10_20 8
LEEF_21_10_22 8
LEEF_21_10_25 8
LEEF_21_10_27 8
LEEF_21_10_29 und 21_11_01 12
LEEF_21_11_03 und 21_11_05 12
LEEF_21_11_08 und 21_11_10 12
LEEF_21_11_12 12
LEEF_21_11_17 und 21_11_19 12
LEEF_21_11_22 und 21_11_24 12
LEEF_21_11_26 und 21_11_29 12
LEEF_21_12_01 und 21_12_03A 12
LEEF_21_12_03B 8
LEEF_21_12_06 und 21_12_08 1-8A 12
LEEF_21_12_08B 4
LEEF_21_12_10 und 21_12_13 12
LEEF_21_12_15 und 21_12_17 12
LEEF_21_12_20 und 21_12_22 12
LEEF_21_12_24 8
LEEF_21_12_27 und 21_12_29 12
LEEF_21_12_31 und 22_01_03 12
LEEF_21_15_10 rest 2 4
LEEF_22_01_05 und 22_01_07A 12
LEEF_22_01_10 und 22_01_12 12
LEEF_22_01_14 und 22_01_17 12
LEEF_22_01_19 und 22_01_21A 4
LEEF_22_01_19 und 22_01_21B 12
LEEF_22_01_24 und 22_01_26 12
LEEF_22_01_28 und 22_01_31A 16
LEEF_22_02_02 und 22_02_04 16
LEEF_22_02_07B 8
LEEF_22_02_07D 8
LEEF_22_02_09 und 22_02_11A 13
LEEF_22_02_14 und 22_02_16A 13
LEEF_22_02_16B 5
LEEF_22_02_18A 12
LEEF_22_02_18B 12
LEEF_22_02_21A 12
LEEF_22_02_23A 12
LEEF_22_02_25 12
LEEF_22_02_28 12
LEEF_22_03_02 12
LEEF_22_03_04 12
LEEF_22_03_07 16
LEEF_22_03_09 16
LEEF_22_03_11 16
LEEF_22_03_14 16
LEEF_22_03_16 16
LEEF_22_03_18 16
LEEF_22_03_21 16
LEEF_22_03_23 16
LEEF_22_03_25 16
LEEF_22_03_28 16
LEEF_22_03_30A 8
LEEF_22_03_30B 8
LEEF_22_04_01 16
LEEF_22_04_04A 8
LEEF_22_04_04B 8
LEEF_22_04_06 16
LEEF_22_04_08 16
LEEF_22_04_11 16
LEEF_22_04_13 16
LEEF_22_04_15A 8
LEEF_22_04_15B 8
LEEF_22_04_18 16
LEEF_22_04_20 16
LEEF_22_04_22 16
LEEF_22_04_25 16
LEEF_22_04_27 16
LEEF_22_04_29 16
LEEF_22_05_02 16
LEEF_22_05_04 16
LEEF_22_05_06 16
LEEF_22_05_09 16
LEEF_22_05_11 16
LEEF_22_05_13 16
LEEF_22_05_16 16
LEEF_22_05_18 16
LEEF_22_05_20 16
LEEF_22_05_23A 16
LEEF_22_05_25A 16
LEEF_22_05_27A 16
LEEF_22_05_30A 16
LEEF_22_06_01A 16
LEEF_22_06_03A 10
LEEF_22_06_06A 10
LEEF_22_06_08A 10
LEEF_22_06_10B 10
LEEF_22_06_13A 10
LEEF_22_06_15A 10
LEEF_22_06_20F 3
LEEF_22_06_22A 8
LEEF_22_06_27A 8
LEEF_22_06_29B 8
LEEF_22_07_04A 8

These can be filtered out

Show the code
unlink(file.path(root_folder, "2.extracted.data", "toc_val_samples.csv"))
toc %>%
  filter(identification == "H2O" | identification == "2.5mg/lIC") %>%
write.csv(file = file.path(root_folder, "2.extracted.data", "toc_val_samples.csv"))
  
before <- nrow(toc)
toc <- toc %>%
  filter(identification != "H2O" & identification != "2.5mg/lIC")
after <- nrow(toc)
cat("Before : ", before, "\n")
Before :  16779 
Show the code
cat("Removed: ", before - after, "\n")
Removed:  1374 
Show the code
cat("After  : ", after, "\n")
After  :  15405 

Check timestamp / id / Filename

In each file, there should be a maximum of 1 timestamp, except of files which contain two timestamps, i.e. which are named accodingly.

Show the code
toc %>%
  select(filename, timestamp, identification, inj_type) %>%
  filter(identification != "H2O" & identification != "2.5mg/lIC") %>%
  group_by(filename, timestamp) %>%
  summarise(timestamps_count = n()) %>%
  group_by(filename) %>%
  summarise(number_of_timestamps = n()) %>%
  arrange(desc(number_of_timestamps)) %>%
  knitr::kable()
filename number_of_timestamps
LEEF_21_09_20 und 21_09_22 C 2
LEEF_21_09_24 und 21_09_27 B 2
LEEF_21_10_04 und 21_10_06A 2
LEEF_21_10_29 und 21_11_01 2
LEEF_21_11_03 und 21_11_05 2
LEEF_21_11_08 und 21_11_10 2
LEEF_21_11_12 2
LEEF_21_11_17 und 21_11_19 2
LEEF_21_11_22 und 21_11_24 2
LEEF_21_11_26 und 21_11_29 2
LEEF_21_12_01 und 21_12_03A 2
LEEF_21_12_06 und 21_12_08 1-8A 2
LEEF_21_12_10 und 21_12_13 2
LEEF_21_12_15 und 21_12_17 2
LEEF_21_12_20 und 21_12_22 2
LEEF_21_12_27 und 21_12_29 2
LEEF_21_12_31 und 22_01_03 2
LEEF_22_01_05 und 22_01_07A 2
LEEF_22_01_10 und 22_01_12 2
LEEF_22_01_14 und 22_01_17 2
LEEF_22_01_19 und 22_01_21B 2
LEEF_22_01_24 und 22_01_26 2
LEEF_22_01_28 und 22_01_31A 2
LEEF_22_02_02 und 22_02_04 2
LEEF_22_02_09 und 22_02_11A 2
LEEF_22_02_14 und 22_02_16A 2
LEEF_ 22_06_17A 1
LEEF_ 22_06_20A 1
LEEF_ 22_06_20B 1
LEEF_21_09_24 1-8 1
LEEF_21_09_27 restB 1
LEEF_21_09_29t 1
LEEF_21_10_01 1
LEEF_21_10_04 S22 - S30 1
LEEF_21_10_06 1
LEEF_21_10_08 1
LEEF_21_10_11 1
LEEF_21_10_13 1
LEEF_21_10_15 1
LEEF_21_10_15 s8 - 26 1
LEEF_21_10_18 1
LEEF_21_10_20 1
LEEF_21_10_22 1
LEEF_21_10_25 1
LEEF_21_10_27 1
LEEF_21_12_03B 1
LEEF_21_12_08B 1
LEEF_21_12_24 1
LEEF_21_15_10 rest 2 1
LEEF_22_01_19 und 22_01_21A 1
LEEF_22_02_07B 1
LEEF_22_02_07D 1
LEEF_22_02_16B 1
LEEF_22_02_18A 1
LEEF_22_02_18B 1
LEEF_22_02_21A 1
LEEF_22_02_23A 1
LEEF_22_02_25 1
LEEF_22_02_28 1
LEEF_22_03_02 1
LEEF_22_03_04 1
LEEF_22_03_07 1
LEEF_22_03_09 1
LEEF_22_03_11 1
LEEF_22_03_14 1
LEEF_22_03_16 1
LEEF_22_03_18 1
LEEF_22_03_21 1
LEEF_22_03_23 1
LEEF_22_03_25 1
LEEF_22_03_28 1
LEEF_22_03_30A 1
LEEF_22_03_30B 1
LEEF_22_04_01 1
LEEF_22_04_04A 1
LEEF_22_04_04B 1
LEEF_22_04_06 1
LEEF_22_04_08 1
LEEF_22_04_11 1
LEEF_22_04_13 1
LEEF_22_04_15A 1
LEEF_22_04_15B 1
LEEF_22_04_18 1
LEEF_22_04_20 1
LEEF_22_04_22 1
LEEF_22_04_25 1
LEEF_22_04_27 1
LEEF_22_04_29 1
LEEF_22_05_02 1
LEEF_22_05_04 1
LEEF_22_05_06 1
LEEF_22_05_09 1
LEEF_22_05_11 1
LEEF_22_05_13 1
LEEF_22_05_16 1
LEEF_22_05_18 1
LEEF_22_05_20 1
LEEF_22_05_23A 1
LEEF_22_05_25A 1
LEEF_22_05_27A 1
LEEF_22_05_30A 1
LEEF_22_06_01A 1
LEEF_22_06_03A 1
LEEF_22_06_06A 1
LEEF_22_06_08A 1
LEEF_22_06_10B 1
LEEF_22_06_13A 1
LEEF_22_06_15A 1
LEEF_22_06_20F 1
LEEF_22_06_22A 1
LEEF_22_06_27A 1
LEEF_22_06_29B 1
LEEF_22_07_04A 1

Fixed the following typos in the raw data. Now, it looks OK for me.

  • LEEF_21_12_10 und 21_12_13: typos in Identification
  • LEEF_22_05_18: typo in Identificatio
  • LEEF_21_11_12: File name should be LEEF_21_11_12 und 21_15_10A
  • LEEF_22_01_19 und 22_01_21A: All analysis for 22_01_21 are not valid

Set unrealistic small TC values to NA

A density plot of the values of the TC measurements in all measurements.

Show the code
tc <- toc %>%
  filter(inj_type == "TC")
tc <- data.frame(conc_TC = c(tc$conc_1, tc$conc_2, tc$conc_3))
tc %>% ggplot(aes(x=conc_TC)) + 
  stat_density(bw = 0.1, na.rm = TRUE) +
  geom_vline(xintercept = 5, col = "red")

One can see for TC two peaks clearly separated. The values smaller than 5 will be set to NA as they are unrealistically low and can be linked to measuring errors in the machine.

Plot the previous plot zoomed in to conc <= 10 for each bottle

conc_1

Show the code
toc %>%
  filter(inj_type == "TC") %>%
  ggplot(aes(x=conc_1)) + 
  geom_vline(xintercept = 5, col = "red") +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")
Warning: Groups with fewer than two data points have been dropped.
Warning: Removed 1 rows containing missing values (`position_stack()`).

conc_2

Show the code
toc %>%
  filter(inj_type == "TC") %>%
  ggplot(aes(x=conc_2)) + 
  geom_vline(xintercept = 5, col = "red") +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")
Warning: Groups with fewer than two data points have been dropped.
Warning: Removed 1 rows containing missing values (`position_stack()`).

conc_3

Show the code
toc %>%
  filter(inj_type == "TC") %>%
  ggplot(aes(x=conc_3)) + 
  geom_vline(xintercept = 5, col = "red") +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

Show the code
tc <- (toc$inj_type == "TC")
toc$conc_1[tc & sapply(toc$conc_1 < 5, isTRUE)] <- NA
toc$conc_2[tc & sapply(toc$conc_2 < 5, isTRUE)] <- NA
toc$conc_3[tc & sapply(toc$conc_3 < 5, isTRUE)] <- NA

Which are now gone

Show the code
tc <- toc %>%
  filter(inj_type == "TC")
tc <- data.frame(conc_TC = c(tc$conc_1, tc$conc_2, tc$conc_3))
tc %>% ggplot(aes(x=conc_TC)) + 
  stat_density(bw = 0.1, na.rm = TRUE) +
  geom_vline(xintercept = 5, col = "red")

Set unrealistic small IC values to NA

A density plot of the values of the IC measurements in all measurements.

Show the code
tc <- toc %>%
  filter(inj_type == "IC")
tc <- data.frame(conc_IC = c(tc$conc_1, tc$conc_2, tc$conc_3))
tc %>% 
  ggplot(aes(x=conc_IC)) + 
  geom_vline(xintercept = 0.2, col = "red") +
  stat_density(bw = 0.001, na.rm = TRUE)

One can see for TC two peaks clearly separated. The values smaller than 0.3 will be set to NA as they are unrealistically low and can be linked to measuring errors in the machine.

Plot the previous plot zoomed in to conc <= 3 for each bottle

conc_1

Show the code
toc %>%
  filter(inj_type == "IC") %>%
  ggplot(aes(x=conc_1)) + 
  geom_vline(xintercept = 0.2, col = "red") +
  stat_density(bw = 0.01, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")
Warning: Groups with fewer than two data points have been dropped.
Warning: Removed 1 rows containing missing values (`position_stack()`).

conc_2

Show the code
toc %>%
  filter(inj_type == "IC") %>%
  ggplot(aes(x=conc_2)) + 
  geom_vline(xintercept = 0.3, col = "red") +
  stat_density(bw = 0.01, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")
Warning: Groups with fewer than two data points have been dropped.
Warning: Removed 1 rows containing missing values (`position_stack()`).

conc_3

Show the code
toc %>%
  filter(inj_type == "IC") %>%
  ggplot(aes(x=conc_3)) + 
  geom_vline(xintercept = 0.3, col = "red") +
  stat_density(bw = 0.01, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

Show the code
tc <- (toc$inj_type == "IC")
toc$conc_1[tc & sapply(toc$conc_1 < 0.3, isTRUE)] <- NA
toc$conc_2[tc & sapply(toc$conc_2 < 0.3, isTRUE)] <- NA
toc$conc_3[tc & sapply(toc$conc_3 < 0.3, isTRUE)] <- NA

Which are now gone

Show the code
tc <- toc %>%
  filter(inj_type == "IC")
tc <- data.frame(conc_IC = c(tc$conc_1, tc$conc_2, tc$conc_3))
tc %>% ggplot(aes(x=conc_IC)) + 
  geom_vline(xintercept = 0.3, col = "red") +
  stat_density(bw = 0.001, na.rm = TRUE)

Recalculate conc and cv for all measurements except TOC

As the number of additional samples as well as the cv were different for some samples, we have to re-calculate conc and cv. The values which will be used are - samples: 2 - extra_samples: 1 - max_cv: 3 In case there are more extra samples, the last ones were be discarded.

As we are re-calculating TOC later, we can exclude the TOC values here.

Re-calculate conc and cv

Show the code
cvf <- function(x, y){
  sapply(
    1:length(x),
    function(i){
      sd(c(x[i], y[i])) / mean(c(x[i], y[i])) * 100
    }
  )
}


toc$cv_12 <- cvf(toc$conc_1, toc$conc_2)
toc$cv_13 <- cvf(toc$conc_1, toc$conc_3)
toc$cv_23 <- cvf(toc$conc_2, toc$conc_3)

toc$conc_12 <- rowMeans(toc[,c("conc_1", "conc_2")])
toc$conc_13 <- rowMeans(toc[,c("conc_1", "conc_3")])
toc$conc_23 <- rowMeans(toc[,c("conc_2", "conc_3")])

for (i in 1:nrow(toc)){
  if (toc[i,"inj_type"] == "TOC"){
    next()
  }
  if (isTRUE(as.numeric(toc[i,][["cv_12"]]) < max_cv)) {111
    toc[i,]$conc <- mean(as.numeric(toc[i,][["conc_1"]]), as.numeric(toc[i,][["conc_2"]]))
    toc[i,]$cv <- as.numeric(toc[i,][["cv_12"]])
  } else {
    sel <- suppressWarnings(
      which(
        toc[i, c("cv_12", "cv_13", "cv_23")] == min(
          as.numeric(toc[i,][["cv_12"]]), 
          as.numeric(toc[i,][["cv_13"]]), 
          as.numeric(toc[i,][["cv_23"]]), 
          na.rm = TRUE
        )
      ) 
    )
    if (length(sel) == 0) {
      toc[i,]$conc <- NA
      toc[i,]$cv <- NA
      next()
    }
    if (((as.numeric(toc[i,][c("conc_1", "conc_2", "conc_3")]) == 0) |> sum(na.rm = TRUE)) > 1) {
      toc[i,]$conc <- NA
      toc[i,]$cv <- NA
      next()
    }
    if (sel == 1) {
      toc[i,]$conc <- mean(as.numeric(toc[i,][["conc_1"]]), as.numeric(toc[i,][["conc_2"]]))
      toc[i,]$cv <- as.numeric(toc[i,][["cv_12"]])
      next()
    }
    if (sel == 2) {
      toc[i,]$conc <- mean(as.numeric(toc[i,][["conc_1"]]), as.numeric(toc[i,][["conc_3"]]))
      toc[i,]$cv <- as.numeric(toc[i,][["cv_13"]])
      next()
    }
    if (sel == 3) {
      toc[i,]$conc <- mean(as.numeric(toc[i,][["conc_2"]]), as.numeric(toc[i,][["conc_3"]]))
      toc[i,]$cv <- as.numeric(toc[i,][["cv_23"]])
      next()
    } else {
      stop("Here is something seriously wrong!!!!")
    }
  }
}

Calculate TOC

Now we re-calculate the TOC values.

Show the code
toc$conc[which(toc$inj_type == "TOC")] <- NA

TOC_calc <- toc %>% 
  mutate(an_id = paste(filename, position)) %>%
  pivot_wider(id_cols = an_id, names_from = inj_type, values_from = conc) %>%
  mutate(TOC_calc = TC - IC) %>%
  select(an_id, TOC_calc) %>%
  mutate(an_id = paste(an_id, "TOC"))

toc <- toc %>% 
  mutate(an_id = paste(filename, position, inj_type)) %>%
  left_join(TOC_calc, by = "an_id") %>%
  mutate( conc = ifelse(inj_type == "TOC", TOC_calc, conc) ) %>%
  select( -TOC_calc, -an_id)

Plot after re-calculation of the TOC values

Show the code
toc %>%   
  ggplot(aes(x=conc)) + 
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_grid(rows = vars(inj_type), scales = "free_y")

Plot the measured concentreation per bottle of all samples

Show the code
toc %>%
  ggplot(aes(x=conc)) + 
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_grid(rows = vars(bottle), cols = vars(inj_type), scales = "free")
Warning: Groups with fewer than two data points have been dropped.
Groups with fewer than two data points have been dropped.
Groups with fewer than two data points have been dropped.
Groups with fewer than two data points have been dropped.
Warning: Removed 4 rows containing missing values (`position_stack()`).

TC

Show the code
toc %>%
  filter(inj_type == "TC") %>%
  ggplot(aes(x=conc)) + 
  geom_vline(xintercept = 5, col = "red") +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")
Warning: Groups with fewer than two data points have been dropped.
Warning: Removed 1 rows containing missing values (`position_stack()`).

IC

Show the code
toc %>%
  filter(inj_type == "IC") %>%
  ggplot(aes(x=conc)) + 
  geom_vline(xintercept = 0.3, col = "red") +
  stat_density(bw = 0.01, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")
Warning: Groups with fewer than two data points have been dropped.
Warning: Removed 1 rows containing missing values (`position_stack()`).

TOC

Show the code
toc %>%
  filter(inj_type == "TOC") %>%
  ggplot(aes(x=conc)) + 
  geom_vline(xintercept = 4.7, col = "red") +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")
Warning: Groups with fewer than two data points have been dropped.
Warning: Removed 1 rows containing missing values (`position_stack()`).

TN

Show the code
toc %>%
  filter(inj_type == "TN") %>%
  ggplot(aes(x=conc)) +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")
Warning: Groups with fewer than two data points have been dropped.
Warning: Removed 1 rows containing missing values (`position_stack()`).

Add to database

The toc is now added to the database

Show the code
add_to_and_overwrite_table_in_RRD(toc = toc, dbname = db)
[1] 0
Show the code
toc_original <- toc

toc$id <- 1:nrow(toc)

Plot after filtering

Now let’s look at the plots of the measurements per bottle per timestep

Show the code
if (params$LEEF == "LEEF_1") {
  p1 <- plot_tocs_per_bottle_per_timestamp(db = db, c("TC", "TOC"))
  p2 <- plot_tocs_per_bottle_per_timestamp(db = db, c("IC"))
  p3 <- plot_tocs_per_bottle_per_timestamp(db = db, c("TN"))
} else {
  p1 <- LEEF_2_plot_tocs_per_bottle_per_timestamp(db = db, c("TC", "TOC"))
  p2 <- LEEF_2_plot_tocs_per_bottle_per_timestamp(db = db, c("IC"))
  p3 <- LEEF_2_plot_tocs_per_bottle_per_timestamp(db = db, c("TN"))
}
Warning: Missing values are always removed in SQL aggregation functions.
Use `na.rm = TRUE` to silence this warning
This warning is displayed once every 8 hours.
Show the code
p1

Show the code
p2

Show the code
p3

Number of measurements per timestamp, bottle and inj_type

We have to look if there are any duplicate measurements.

Show the code
dupl <- toc %>%
  group_by(timestamp, bottle, inj_type) %>%
  filter(!is.na(bottle)) %>%
  summarise(fn_1 = min(filename), fn_2 = max(filename), count = n(), min_conc = min(conc), max_conc = max(conc)) %>%
  filter(count > 1 & count < 10) %>%  
  mutate(diff = max_conc - min_conc) %>%
  arrange(inj_type, diff)
dupl %>% 
  group_by(fn_1, fn_2, timestamp) %>%
  summarise() %>%
  arrange(timestamp) %>%
  knitr::kable()
fn_1 fn_2 timestamp
LEEF_21_09_24 1-8 LEEF_21_09_24 und 21_09_27 B 20210924
LEEF_21_09_24 und 21_09_27 B LEEF_21_09_27 restB 20210927
LEEF_21_10_04 S22 - S30 LEEF_21_10_04 und 21_10_06A 20211004
LEEF_21_10_04 und 21_10_06A LEEF_21_10_06 20211006
LEEF_21_10_01 LEEF_21_10_29 und 21_11_01 20211101
LEEF_21_10_15 s8 - 26 LEEF_21_11_12 20211115
LEEF_21_10_15 s8 - 26 LEEF_21_15_10 rest 2 20211115
LEEF_21_12_01 und 21_12_03A LEEF_21_12_03B 20211203
LEEF_21_12_06 und 21_12_08 1-8A LEEF_21_12_08B 20211208
LEEF_22_01_19 und 22_01_21A LEEF_22_01_19 und 22_01_21B 20220119
LEEF_22_02_07B LEEF_22_02_07D 20220207
LEEF_22_02_14 und 22_02_16A LEEF_22_02_16B 20220216
LEEF_22_02_18A LEEF_22_02_18B 20220218
LEEF_22_03_30A LEEF_22_03_30B 20220330
LEEF_22_04_04A LEEF_22_04_04B 20220404
LEEF_22_04_15A LEEF_22_04_15B 20220415
LEEF_ 22_06_20A LEEF_ 22_06_20B 20220620

And some plots of the duplicate concentration values only

Show the code
dat <- db_read_toc(db = db, duplicates = NULL) %>% 
  collect()
if (nrow(dat) > 0){
dat$id <- 1:nrow(dat)
ids <- dat %>% 
  filter(!is.na(bottle)) %>%
  group_by(timestamp, bottle, type) %>%
  summarize(min(id), max(id), n = n()) %>%
  filter(n > 1)
ids <- c(ids$`min(id)`, ids$`max(id)`) |>
  unique()
x <- dat %>% 
  filter(id %in% ids) %>%
  group_by(day, timestamp, bottle, type) %>%
  summarise(timestamp, bottle, type, mic = min(concentration), mac = max(concentration)) %>%
  mutate(mdiff = mac - mic)
}

TN Duplicates

Show the code
if (nrow(x) > 0){
  pl <- x %>%
    filter(type == "TN") %>%
    ggplot2::ggplot(ggplot2::aes(x = mic, y = mac, colour = type)) +
    ggplot2::geom_point() +
    ggplot2::xlab("Smaler concentration Value") +
    ggplot2::ylab("Larger concentration Value")
  
  ggMarginal(pl, type="histogram")
}
Warning: Removed 308 rows containing missing values (`geom_point()`).

TC Duplicates

Show the code
if (nrow(x) > 0){
  pl <- x %>%
    filter(type == "TC") %>%
    ggplot2::ggplot(ggplot2::aes(x = mic, y = mac, colour = type)) +
    ggplot2::geom_point() +
    ggplot2::xlab("Smaler concentration Value") +
    ggplot2::ylab("Larger concentration Value")
  
  ggMarginal(pl, type="histogram")
}
Warning: Removed 306 rows containing missing values (`geom_point()`).

IC Duplicates

Show the code
if (nrow(x) > 0){
  pl <- x %>%
    filter(type == "IC") %>%
    ggplot2::ggplot(ggplot2::aes(x = mic, y = mac, colour = type)) +
    ggplot2::geom_point() +
    ggplot2::xlab("Smaler concentration Value") +
    ggplot2::ylab("Larger concentration Value")
  
  ggMarginal(pl, type="histogram")
}
Warning: Removed 316 rows containing missing values (`geom_point()`).

TOC Duplicates

Show the code
if (nrow(x) > 0){
  pl <- x %>%
    filter(type == "TOC") %>%
    ggplot2::ggplot(ggplot2::aes(x = mic, y = mac, colour = type)) +
    ggplot2::geom_point() +
    ggplot2::xlab("Smaler concentration Value") +
    ggplot2::ylab("Larger concentration Value")
  
  ggMarginal(pl, type="histogram")
}
Warning: Removed 318 rows containing missing values (`geom_point()`).

Final diagnostic report

Show the code
try(
  report_diagnostic(
    db = params$db,  
    template = params$LEEF,
    suffix = "TOC_added", 
    format = "html"
  )
)


processing file: DiagnosticReport.Rmd
Error in parse_block(g[-1], g[1], params.src, markdown_mode) : 
  Duplicate chunk label 'setup', which has been used for the chunk:
library(LEEF.analysis)
library(LEEF.measurement.toc)
library(dplyr)
library(tidyr)
library(ggplot2)
library(ggExtra)
options(dplyr.summarise.inform = FALSE)